unique_keywords = df['searchkeyword'].unique()
def calculate_metrics(embeddings):
pairwise_distances = pdist(embeddings, metric='cosine')
avg_pairwise_distance = pairwise_distances.mean()
knn = NearestNeighbors(n_neighbors=12)
knn.fit(embeddings)
distances, _ = knn.kneighbors(embeddings)
avg_local_density = distances.mean()
multivariate_std = np.std(embeddings, axis=0).mean()
return avg_pairwise_distance, avg_local_density, multivariate_std
with open('../data/raw/coins.json', 'r') as f:
coins_data = json.load(f)
coins_info_df = pd.DataFrame(coins_data)
merged_df = df.merge(coins_info_df[['name', 'fraud']], left_on="searchkeyword", right_on="name", how="left")
if 'fraud' not in merged_df.columns:
print("Die 'fraud'-Spalte konnte nicht hinzugefügt werden. Bitte überprüfen Sie die Zuordnung.")
unique_keywords = merged_df['searchkeyword'].unique()
metrics_data = []
for keyword in unique_keywords:
keyword_df = merged_df[merged_df['searchkeyword'] == keyword]
fraud_values = keyword_df[keyword_df['fraud'] == True]['embedding'].values
non_fraud_values = keyword_df[keyword_df['fraud'] == False]['embedding'].values
if len(fraud_values) > 0:
fraud_embeddings = np.vstack(fraud_values)
fraud_metrics = calculate_metrics(fraud_embeddings)
else:
fraud_metrics = (0, 0, 0)
if len(non_fraud_values) > 0:
non_fraud_embeddings = np.vstack(non_fraud_values)
non_fraud_metrics = calculate_metrics(non_fraud_embeddings)
else:
non_fraud_metrics = (0, 0, 0)
metrics_data.append({
'keyword': keyword,
'fraud': fraud_metrics,
'non_fraud': non_fraud_metrics
})
fig, axes = plt.subplots(3, 1, figsize=(12, 18))
metric_names = ['Average Pairwise Distance', 'Average Local Density', 'Multivariate Standard Deviation']
for i, metric_name in enumerate(metric_names):
fraud_values = [entry['fraud'][i] for entry in metrics_data]
non_fraud_values = [entry['non_fraud'][i] for entry in metrics_data]
keywords = [entry['keyword'] for entry in metrics_data]
bar_width = 0.4
x = np.arange(len(keywords))
axes[i].bar(x - bar_width / 2, fraud_values, width=bar_width, label='Fraud', color='red', alpha=0.7)
axes[i].bar(x + bar_width / 2, non_fraud_values, width=bar_width, label='Non-Fraud', color='blue', alpha=0.7)
axes[i].set_title(metric_name)
axes[i].set_xticks(x)
axes[i].set_xticklabels(keywords, rotation=45, ha='right')
axes[i].set_ylabel('Value')
axes[i].legend()
plt.tight_layout()
plt.show()
